# -*- coding:utf-8 _*-
import requests
import os
from urllib import parse
from lxml import etree
from fake_useragent import UserAgent

api = 'http://s.huanqiu.com/s/?q='

def format(string):
    # 文件命名不能出现 \ / : * ? " < > |
    string = str(string).replace('\\', '')
    string = (((string.replace('/', '')).replace(':', '：')).replace('*', '')).replace('?', '？')
    string = (((string.replace('"', '“')).replace('<', '《')).replace('>', '》')).replace('|', '')
    return string

def Scan():
    dir_list = os.listdir('./txt')
    file_list = []
    for dir in dir_list:
        file_list.append(dir.strip('.txt'))
    return file_list

def get_data(keyword):
    try:
        main_url = api + parse.quote(keyword) + '&p='
        response = requests.get(url=main_url, headers={'User-Agent': UserAgent().random}).text
        html = etree.HTML(response)
        total_list = html.xpath('/html/body/div[2]/div/div/div[1]/div[1]/em[1]/text()')
        total = total_list[0] if len(total_list) != 0 else None
        if total is not None:
            if int(total) % 10 == 0:
                total_page = int(total) / 10
            else:
                total_page = int(total) // 10 + 1
            for i in range(1, total_page + 1):
                next_url = main_url + str(i)
                try:
                    response = requests.get(url=next_url, headers={'User-Agent': UserAgent().random}).text
                    html = etree.HTML(response)
                    detail_url_list = html.xpath('//div[@class="sList"]/ul/li/a/@href')
                    if len(detail_url_list) != 0:
                        for detail_url in detail_url_list:
                            try:
                                res = requests.get(url=detail_url, headers={'User-Agent': UserAgent().random})
                                res.encoding = 'utf-8'
                                response = res.text
                                html = etree.HTML(response)
                                title_list = html.xpath('//h1/text()')
                                p_list = html.xpath('//div[@class="text"]/p/text()')
                                title = format(title_list[0]) if len(title_list) != 0 else None
                                if title is not None:
                                    file_list =Scan()
                                    if title in file_list:  # 存在是Ture
                                        pass
                                    else:  # 不存在False
                                        content_list = []
                                        for p in p_list:
                                            p = (((str(p).replace('\r\n', '')).replace(' ', '')).replace('\u3000', '')).replace(
                                                '\n', '')
                                            p = p.replace('环球网', '发布网站')
                                            if p != '':
                                                content_list.append(p)
                                        if title is not None and len(content_list) != 0:
                                            with open('./txt/{}.txt'.format(title), 'a+', encoding='utf-8') as f:
                                                f.write('<title={}>'.format(title) + '\n')
                                            with open('./txt/{}.txt'.format(title), 'a+', encoding='utf-8') as f:
                                                f.write('<neirong='+'\n')
                                            for content in content_list:
                                                with open('./txt/{}.txt'.format(title), 'a+', encoding='utf-8') as f:
                                                    f.write('<p>{}</p>'.format(content) + '\n')
                                            with open('./txt/{}.txt'.format(title), 'a+', encoding='utf-8') as f:
                                                f.write('>')
                                            print(title, '     完成写入..........')
                            except:
                                pass
                except:
                    pass
    except:
        pass

def run():
    keyword = '雅思'
    get_data(keyword)

if __name__ == '__main__':
    run()
